

<!DOCTYPE html>
<html class="writer-html5" lang="zh" >
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>常用做法 &mdash; Scrapy 2.3.0 文档</title>
  

  
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster.custom.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster.bundle.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-shadow.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-punk.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-noir.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-light.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-borderless.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/micromodal.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/sphinx_rtd_theme.css" type="text/css" />

  
  
  
  

  
  <!--[if lt IE 9]>
    <script src="../_static/js/html5shiv.min.js"></script>
  <![endif]-->
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
        <script src="../_static/jquery.js"></script>
        <script src="../_static/underscore.js"></script>
        <script src="../_static/doctools.js"></script>
        <script src="../_static/language_data.js"></script>
        <script src="../_static/js/hoverxref.js"></script>
        <script src="../_static/js/tooltipster.bundle.min.js"></script>
        <script src="../_static/js/micromodal.min.js"></script>
    
    <script type="text/javascript" src="../_static/js/theme.js"></script>

    
    <link rel="index" title="索引" href="../genindex.html" />
    <link rel="search" title="搜索" href="../search.html" />
    <link rel="next" title="宽爬行" href="broad-crawls.html" />
    <link rel="prev" title="蜘蛛合约" href="contracts.html" /> 
</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

          
            <a href="../index.html" class="icon icon-home" alt="Documentation Home"> Scrapy
          

          
          </a>

          
            
            
              <div class="version">
                2.3
              </div>
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        
        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption"><span class="caption-text">第一步</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../intro/overview.html">Scrapy一目了然</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intro/install.html">安装指南</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intro/tutorial.html">Scrapy 教程</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intro/examples.html">实例</a></li>
</ul>
<p class="caption"><span class="caption-text">基本概念</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="commands.html">命令行工具</a></li>
<li class="toctree-l1"><a class="reference internal" href="spiders.html">蜘蛛</a></li>
<li class="toctree-l1"><a class="reference internal" href="selectors.html">选择器</a></li>
<li class="toctree-l1"><a class="reference internal" href="items.html">项目</a></li>
<li class="toctree-l1"><a class="reference internal" href="loaders.html">项目加载器</a></li>
<li class="toctree-l1"><a class="reference internal" href="shell.html">Scrapy shell</a></li>
<li class="toctree-l1"><a class="reference internal" href="item-pipeline.html">项目管道</a></li>
<li class="toctree-l1"><a class="reference internal" href="feed-exports.html">Feed 导出</a></li>
<li class="toctree-l1"><a class="reference internal" href="request-response.html">请求和响应</a></li>
<li class="toctree-l1"><a class="reference internal" href="link-extractors.html">链接提取器</a></li>
<li class="toctree-l1"><a class="reference internal" href="settings.html">设置</a></li>
<li class="toctree-l1"><a class="reference internal" href="exceptions.html">例外情况</a></li>
</ul>
<p class="caption"><span class="caption-text">内置服务</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="logging.html">登录</a></li>
<li class="toctree-l1"><a class="reference internal" href="stats.html">统计数据集合</a></li>
<li class="toctree-l1"><a class="reference internal" href="email.html">发送电子邮件</a></li>
<li class="toctree-l1"><a class="reference internal" href="telnetconsole.html">远程登录控制台</a></li>
<li class="toctree-l1"><a class="reference internal" href="webservice.html">Web服务</a></li>
</ul>
<p class="caption"><span class="caption-text">解决具体问题</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../faq.html">常见问题</a></li>
<li class="toctree-l1"><a class="reference internal" href="debug.html">调试spiders</a></li>
<li class="toctree-l1"><a class="reference internal" href="contracts.html">蜘蛛合约</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">常用做法</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#run-scrapy-from-a-script">从脚本中运行Scrapy</a></li>
<li class="toctree-l2"><a class="reference internal" href="#running-multiple-spiders-in-the-same-process">在同一进程中运行多个spider</a></li>
<li class="toctree-l2"><a class="reference internal" href="#distributed-crawls">分布式爬行</a></li>
<li class="toctree-l2"><a class="reference internal" href="#avoiding-getting-banned">避免被禁止</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="broad-crawls.html">宽爬行</a></li>
<li class="toctree-l1"><a class="reference internal" href="developer-tools.html">使用浏览器的开发人员工具进行抓取</a></li>
<li class="toctree-l1"><a class="reference internal" href="dynamic-content.html">选择动态加载的内容</a></li>
<li class="toctree-l1"><a class="reference internal" href="leaks.html">调试内存泄漏</a></li>
<li class="toctree-l1"><a class="reference internal" href="media-pipeline.html">下载和处理文件和图像</a></li>
<li class="toctree-l1"><a class="reference internal" href="deploy.html">部署蜘蛛</a></li>
<li class="toctree-l1"><a class="reference internal" href="autothrottle.html">AutoThrottle 扩展</a></li>
<li class="toctree-l1"><a class="reference internal" href="benchmarking.html">标杆管理</a></li>
<li class="toctree-l1"><a class="reference internal" href="jobs.html">作业：暂停和恢复爬行</a></li>
<li class="toctree-l1"><a class="reference internal" href="coroutines.html">协同程序</a></li>
<li class="toctree-l1"><a class="reference internal" href="asyncio.html">asyncio</a></li>
</ul>
<p class="caption"><span class="caption-text">扩展Scrapy</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="architecture.html">体系结构概述</a></li>
<li class="toctree-l1"><a class="reference internal" href="downloader-middleware.html">下载器中间件</a></li>
<li class="toctree-l1"><a class="reference internal" href="spider-middleware.html">蜘蛛中间件</a></li>
<li class="toctree-l1"><a class="reference internal" href="extensions.html">扩展</a></li>
<li class="toctree-l1"><a class="reference internal" href="api.html">核心API</a></li>
<li class="toctree-l1"><a class="reference internal" href="signals.html">信号</a></li>
<li class="toctree-l1"><a class="reference internal" href="exporters.html">条目导出器</a></li>
</ul>
<p class="caption"><span class="caption-text">其余所有</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../news.html">发行说明</a></li>
<li class="toctree-l1"><a class="reference internal" href="../contributing.html">为 Scrapy 贡献</a></li>
<li class="toctree-l1"><a class="reference internal" href="../versioning.html">版本控制和API稳定性</a></li>
</ul>

            
          
        </div>
        
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../index.html">Scrapy</a>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../index.html" class="icon icon-home"></a> &raquo;</li>
        
      <li>常用做法</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="common-practices">
<span id="topics-practices"></span><h1>常用做法<a class="headerlink" href="#common-practices" title="永久链接至标题">¶</a></h1>
<p>本节记录使用Scrapy时的常见做法。这些内容涵盖了许多主题，通常不属于任何其他特定部分。</p>
<div class="section" id="run-scrapy-from-a-script">
<span id="run-from-script"></span><h2>从脚本中运行Scrapy<a class="headerlink" href="#run-scrapy-from-a-script" title="永久链接至标题">¶</a></h2>
<p>你可以使用 <a class="reference internal" href="api.html#topics-api"><span class="std std-ref">API</span></a> 从脚本运行scrapy，而不是运行scrapy via的典型方式 <code class="docutils literal notranslate"><span class="pre">scrapy</span> <span class="pre">crawl</span></code> .</p><script async src="https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js"></script>
<ins class="adsbygoogle"
     style="display:block; text-align:center;"
     data-ad-layout="in-article"
     data-ad-format="fluid"
     data-ad-client="ca-pub-1466963416408457"
     data-ad-slot="8850786025"></ins>
<script>
     (adsbygoogle = window.adsbygoogle || []).push({});
</script>
<p>记住，scrappy构建在TwistedAsynchronicNetworkLibrary之上，所以需要在TwistedReactor中运行它。</p>
<p>你能用来运行蜘蛛的第一个工具是 <code class="xref py py-class docutils literal notranslate"><span class="pre">scrapy.crawler.CrawlerProcess</span></code> . 这个类将为您启动一个扭曲的反应器，配置日志记录和设置关闭处理程序。这个类是所有slapy命令使用的类。</p>
<p>下面是一个示例，演示如何使用它运行单个蜘蛛。</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">scrapy</span>
<span class="kn">from</span> <span class="nn">scrapy.crawler</span> <span class="kn">import</span> <span class="n">CrawlerProcess</span>

<span class="k">class</span> <span class="nc">MySpider</span><span class="p">(</span><span class="n">scrapy</span><span class="o">.</span><span class="n">Spider</span><span class="p">):</span>
    <span class="c1"># Your spider definition</span>
    <span class="o">...</span>

<span class="n">process</span> <span class="o">=</span> <span class="n">CrawlerProcess</span><span class="p">(</span><span class="n">settings</span><span class="o">=</span><span class="p">{</span>
    <span class="s2">&quot;FEEDS&quot;</span><span class="p">:</span> <span class="p">{</span>
        <span class="s2">&quot;items.json&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;format&quot;</span><span class="p">:</span> <span class="s2">&quot;json&quot;</span><span class="p">},</span>
    <span class="p">},</span>
<span class="p">})</span>

<span class="n">process</span><span class="o">.</span><span class="n">crawl</span><span class="p">(</span><span class="n">MySpider</span><span class="p">)</span>
<span class="n">process</span><span class="o">.</span><span class="n">start</span><span class="p">()</span> <span class="c1"># the script will block here until the crawling is finished</span>
</pre></div>
</div>
<p>在CrawlerProcess中定义字典中的设置。一定要检查 <code class="xref py py-class docutils literal notranslate"><span class="pre">CrawlerProcess</span></code> 了解其使用细节的文档。</p>
<p>如果您在一个零碎的项目中，有一些额外的帮助器可以用来导入项目中的那些组件。你可以自动输入蜘蛛的名字 <code class="xref py py-class docutils literal notranslate"><span class="pre">CrawlerProcess</span></code> 及使用 <code class="docutils literal notranslate"><span class="pre">get_project_settings</span></code> 得到一个 <a class="reference internal" href="api.html#scrapy.settings.Settings" title="scrapy.settings.Settings"><code class="xref py py-class docutils literal notranslate"><span class="pre">Settings</span></code></a> 具有项目设置的实例。</p>
<p>下面是一个如何使用 <a class="reference external" href="https://github.com/scrapinghub/testspiders">testspiders</a> 以项目为例。</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">scrapy.crawler</span> <span class="kn">import</span> <span class="n">CrawlerProcess</span>
<span class="kn">from</span> <span class="nn">scrapy.utils.project</span> <span class="kn">import</span> <span class="n">get_project_settings</span>

<span class="n">process</span> <span class="o">=</span> <span class="n">CrawlerProcess</span><span class="p">(</span><span class="n">get_project_settings</span><span class="p">())</span>

<span class="c1"># &#39;followall&#39; is the name of one of the spiders of the project.</span>
<span class="n">process</span><span class="o">.</span><span class="n">crawl</span><span class="p">(</span><span class="s1">&#39;followall&#39;</span><span class="p">,</span> <span class="n">domain</span><span class="o">=</span><span class="s1">&#39;scrapinghub.com&#39;</span><span class="p">)</span>
<span class="n">process</span><span class="o">.</span><span class="n">start</span><span class="p">()</span> <span class="c1"># the script will block here until the crawling is finished</span>
</pre></div>
</div>
<p>还有另一个Scrapy实用程序，它提供了对爬行过程的更多控制： <code class="xref py py-class docutils literal notranslate"><span class="pre">scrapy.crawler.CrawlerRunner</span></code> . 这个类是一个薄包装器，它封装了一些简单的帮助器来运行多个爬行器，但是它不会以任何方式启动或干扰现有的反应器。</p>
<p>使用这个类，在调度spider之后应该显式地运行reactor。建议您使用 <code class="xref py py-class docutils literal notranslate"><span class="pre">CrawlerRunner</span></code> 而不是 <code class="xref py py-class docutils literal notranslate"><span class="pre">CrawlerProcess</span></code> 如果您的应用程序已经在使用Twisted，并且您希望在同一个反应器中运行Scrapy。</p>
<p>请注意，蜘蛛完成后，您还必须自己关闭扭曲的反应堆。这可以通过将回调添加到由 <code class="xref py py-meth docutils literal notranslate"><span class="pre">CrawlerRunner.crawl</span></code> 方法。</p>
<p>下面是一个使用它的例子，以及在 <code class="docutils literal notranslate"><span class="pre">MySpider</span></code> 已完成运行。</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">twisted.internet</span> <span class="kn">import</span> <span class="n">reactor</span>
<span class="kn">import</span> <span class="nn">scrapy</span>
<span class="kn">from</span> <span class="nn">scrapy.crawler</span> <span class="kn">import</span> <span class="n">CrawlerRunner</span>
<span class="kn">from</span> <span class="nn">scrapy.utils.log</span> <span class="kn">import</span> <span class="n">configure_logging</span>

<span class="k">class</span> <span class="nc">MySpider</span><span class="p">(</span><span class="n">scrapy</span><span class="o">.</span><span class="n">Spider</span><span class="p">):</span>
    <span class="c1"># Your spider definition</span>
    <span class="o">...</span>

<span class="n">configure_logging</span><span class="p">({</span><span class="s1">&#39;LOG_FORMAT&#39;</span><span class="p">:</span> <span class="s1">&#39;</span><span class="si">%(levelname)s</span><span class="s1">: </span><span class="si">%(message)s</span><span class="s1">&#39;</span><span class="p">})</span>
<span class="n">runner</span> <span class="o">=</span> <span class="n">CrawlerRunner</span><span class="p">()</span>

<span class="n">d</span> <span class="o">=</span> <span class="n">runner</span><span class="o">.</span><span class="n">crawl</span><span class="p">(</span><span class="n">MySpider</span><span class="p">)</span>
<span class="n">d</span><span class="o">.</span><span class="n">addBoth</span><span class="p">(</span><span class="k">lambda</span> <span class="n">_</span><span class="p">:</span> <span class="n">reactor</span><span class="o">.</span><span class="n">stop</span><span class="p">())</span>
<span class="n">reactor</span><span class="o">.</span><span class="n">run</span><span class="p">()</span> <span class="c1"># the script will block here until the crawling is finished</span>
</pre></div>
</div>
<div class="admonition seealso">
<p class="admonition-title">参见</p>
<p><a class="reference external" href="https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html" title="(在 Twisted v20.3)"><span>Reactor Overview</span></a></p>
</div>
</div>
<div class="section" id="running-multiple-spiders-in-the-same-process">
<span id="run-multiple-spiders"></span><h2>在同一进程中运行多个spider<a class="headerlink" href="#running-multiple-spiders-in-the-same-process" title="永久链接至标题">¶</a></h2>
<p>默认情况下，当您运行时，scrapy为每个进程运行一个spider <code class="docutils literal notranslate"><span class="pre">scrapy</span> <span class="pre">crawl</span></code> . 但是，Scrapy支持使用 <a class="reference internal" href="api.html#topics-api"><span class="std std-ref">internal API</span></a> .</p>
<p>下面是一个同时运行多个蜘蛛的示例：</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">scrapy</span>
<span class="kn">from</span> <span class="nn">scrapy.crawler</span> <span class="kn">import</span> <span class="n">CrawlerProcess</span>

<span class="k">class</span> <span class="nc">MySpider1</span><span class="p">(</span><span class="n">scrapy</span><span class="o">.</span><span class="n">Spider</span><span class="p">):</span>
    <span class="c1"># Your first spider definition</span>
    <span class="o">...</span>

<span class="k">class</span> <span class="nc">MySpider2</span><span class="p">(</span><span class="n">scrapy</span><span class="o">.</span><span class="n">Spider</span><span class="p">):</span>
    <span class="c1"># Your second spider definition</span>
    <span class="o">...</span>

<span class="n">process</span> <span class="o">=</span> <span class="n">CrawlerProcess</span><span class="p">()</span>
<span class="n">process</span><span class="o">.</span><span class="n">crawl</span><span class="p">(</span><span class="n">MySpider1</span><span class="p">)</span>
<span class="n">process</span><span class="o">.</span><span class="n">crawl</span><span class="p">(</span><span class="n">MySpider2</span><span class="p">)</span>
<span class="n">process</span><span class="o">.</span><span class="n">start</span><span class="p">()</span> <span class="c1"># the script will block here until all crawling jobs are finished</span>
</pre></div>
</div>
<p>使用相同的示例 <code class="xref py py-class docutils literal notranslate"><span class="pre">CrawlerRunner</span></code> ：</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">scrapy</span>
<span class="kn">from</span> <span class="nn">twisted.internet</span> <span class="kn">import</span> <span class="n">reactor</span>
<span class="kn">from</span> <span class="nn">scrapy.crawler</span> <span class="kn">import</span> <span class="n">CrawlerRunner</span>
<span class="kn">from</span> <span class="nn">scrapy.utils.log</span> <span class="kn">import</span> <span class="n">configure_logging</span>

<span class="k">class</span> <span class="nc">MySpider1</span><span class="p">(</span><span class="n">scrapy</span><span class="o">.</span><span class="n">Spider</span><span class="p">):</span>
    <span class="c1"># Your first spider definition</span>
    <span class="o">...</span>

<span class="k">class</span> <span class="nc">MySpider2</span><span class="p">(</span><span class="n">scrapy</span><span class="o">.</span><span class="n">Spider</span><span class="p">):</span>
    <span class="c1"># Your second spider definition</span>
    <span class="o">...</span>

<span class="n">configure_logging</span><span class="p">()</span>
<span class="n">runner</span> <span class="o">=</span> <span class="n">CrawlerRunner</span><span class="p">()</span>
<span class="n">runner</span><span class="o">.</span><span class="n">crawl</span><span class="p">(</span><span class="n">MySpider1</span><span class="p">)</span>
<span class="n">runner</span><span class="o">.</span><span class="n">crawl</span><span class="p">(</span><span class="n">MySpider2</span><span class="p">)</span>
<span class="n">d</span> <span class="o">=</span> <span class="n">runner</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
<span class="n">d</span><span class="o">.</span><span class="n">addBoth</span><span class="p">(</span><span class="k">lambda</span> <span class="n">_</span><span class="p">:</span> <span class="n">reactor</span><span class="o">.</span><span class="n">stop</span><span class="p">())</span>

<span class="n">reactor</span><span class="o">.</span><span class="n">run</span><span class="p">()</span> <span class="c1"># the script will block here until all crawling jobs are finished</span>
</pre></div>
</div>
<p>同样的例子，但是通过链接延迟来按顺序运行spider：</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">twisted.internet</span> <span class="kn">import</span> <span class="n">reactor</span><span class="p">,</span> <span class="n">defer</span>
<span class="kn">from</span> <span class="nn">scrapy.crawler</span> <span class="kn">import</span> <span class="n">CrawlerRunner</span>
<span class="kn">from</span> <span class="nn">scrapy.utils.log</span> <span class="kn">import</span> <span class="n">configure_logging</span>

<span class="k">class</span> <span class="nc">MySpider1</span><span class="p">(</span><span class="n">scrapy</span><span class="o">.</span><span class="n">Spider</span><span class="p">):</span>
    <span class="c1"># Your first spider definition</span>
    <span class="o">...</span>

<span class="k">class</span> <span class="nc">MySpider2</span><span class="p">(</span><span class="n">scrapy</span><span class="o">.</span><span class="n">Spider</span><span class="p">):</span>
    <span class="c1"># Your second spider definition</span>
    <span class="o">...</span>

<span class="n">configure_logging</span><span class="p">()</span>
<span class="n">runner</span> <span class="o">=</span> <span class="n">CrawlerRunner</span><span class="p">()</span>

<span class="nd">@defer</span><span class="o">.</span><span class="n">inlineCallbacks</span>
<span class="k">def</span> <span class="nf">crawl</span><span class="p">():</span>
    <span class="k">yield</span> <span class="n">runner</span><span class="o">.</span><span class="n">crawl</span><span class="p">(</span><span class="n">MySpider1</span><span class="p">)</span>
    <span class="k">yield</span> <span class="n">runner</span><span class="o">.</span><span class="n">crawl</span><span class="p">(</span><span class="n">MySpider2</span><span class="p">)</span>
    <span class="n">reactor</span><span class="o">.</span><span class="n">stop</span><span class="p">()</span>

<span class="n">crawl</span><span class="p">()</span>
<span class="n">reactor</span><span class="o">.</span><span class="n">run</span><span class="p">()</span> <span class="c1"># the script will block here until the last crawl call is finished</span>
</pre></div>
</div>
<div class="admonition seealso">
<p class="admonition-title">参见</p>
<p><a class="reference internal" href="#run-from-script"><span class="std std-ref">从脚本中运行Scrapy</span></a> .</p>
</div>
</div>
<div class="section" id="distributed-crawls">
<span id="id1"></span><h2>分布式爬行<a class="headerlink" href="#distributed-crawls" title="永久链接至标题">¶</a></h2>
<p>Scrapy不提供任何以分布式（多服务器）方式运行爬虫的内置工具。但是，有一些分发爬行的方法，这取决于您计划如何分发爬行。</p>
<p>如果您有许多蜘蛛，那么分配负载的最明显的方法就是设置许多ScrapyD实例，并将蜘蛛运行分布在这些实例中。</p>
<p>如果您想在多台机器上运行一个（大）蜘蛛，通常需要对URL进行分区，以便爬行并将它们发送到每个单独的蜘蛛。下面是一个具体的例子：</p>
<p>首先，准备要爬网的URL列表并将其放入单独的文件/URL:：</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">http</span><span class="p">:</span><span class="o">//</span><span class="n">somedomain</span><span class="o">.</span><span class="n">com</span><span class="o">/</span><span class="n">urls</span><span class="o">-</span><span class="n">to</span><span class="o">-</span><span class="n">crawl</span><span class="o">/</span><span class="n">spider1</span><span class="o">/</span><span class="n">part1</span><span class="o">.</span><span class="n">list</span>
<span class="n">http</span><span class="p">:</span><span class="o">//</span><span class="n">somedomain</span><span class="o">.</span><span class="n">com</span><span class="o">/</span><span class="n">urls</span><span class="o">-</span><span class="n">to</span><span class="o">-</span><span class="n">crawl</span><span class="o">/</span><span class="n">spider1</span><span class="o">/</span><span class="n">part2</span><span class="o">.</span><span class="n">list</span>
<span class="n">http</span><span class="p">:</span><span class="o">//</span><span class="n">somedomain</span><span class="o">.</span><span class="n">com</span><span class="o">/</span><span class="n">urls</span><span class="o">-</span><span class="n">to</span><span class="o">-</span><span class="n">crawl</span><span class="o">/</span><span class="n">spider1</span><span class="o">/</span><span class="n">part3</span><span class="o">.</span><span class="n">list</span>
</pre></div>
</div>
<p>然后在3个不同的ScrapyD服务器上启动一个蜘蛛运行。蜘蛛会收到一个（蜘蛛）论点 <code class="docutils literal notranslate"><span class="pre">part</span></code> 使用要爬网的分区的编号：：</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">curl</span> <span class="n">http</span><span class="p">:</span><span class="o">//</span><span class="n">scrapy1</span><span class="o">.</span><span class="n">mycompany</span><span class="o">.</span><span class="n">com</span><span class="p">:</span><span class="mi">6800</span><span class="o">/</span><span class="n">schedule</span><span class="o">.</span><span class="n">json</span> <span class="o">-</span><span class="n">d</span> <span class="n">project</span><span class="o">=</span><span class="n">myproject</span> <span class="o">-</span><span class="n">d</span> <span class="n">spider</span><span class="o">=</span><span class="n">spider1</span> <span class="o">-</span><span class="n">d</span> <span class="n">part</span><span class="o">=</span><span class="mi">1</span>
<span class="n">curl</span> <span class="n">http</span><span class="p">:</span><span class="o">//</span><span class="n">scrapy2</span><span class="o">.</span><span class="n">mycompany</span><span class="o">.</span><span class="n">com</span><span class="p">:</span><span class="mi">6800</span><span class="o">/</span><span class="n">schedule</span><span class="o">.</span><span class="n">json</span> <span class="o">-</span><span class="n">d</span> <span class="n">project</span><span class="o">=</span><span class="n">myproject</span> <span class="o">-</span><span class="n">d</span> <span class="n">spider</span><span class="o">=</span><span class="n">spider1</span> <span class="o">-</span><span class="n">d</span> <span class="n">part</span><span class="o">=</span><span class="mi">2</span>
<span class="n">curl</span> <span class="n">http</span><span class="p">:</span><span class="o">//</span><span class="n">scrapy3</span><span class="o">.</span><span class="n">mycompany</span><span class="o">.</span><span class="n">com</span><span class="p">:</span><span class="mi">6800</span><span class="o">/</span><span class="n">schedule</span><span class="o">.</span><span class="n">json</span> <span class="o">-</span><span class="n">d</span> <span class="n">project</span><span class="o">=</span><span class="n">myproject</span> <span class="o">-</span><span class="n">d</span> <span class="n">spider</span><span class="o">=</span><span class="n">spider1</span> <span class="o">-</span><span class="n">d</span> <span class="n">part</span><span class="o">=</span><span class="mi">3</span>
</pre></div>
</div>
</div>
<div class="section" id="avoiding-getting-banned">
<span id="bans"></span><h2>避免被禁止<a class="headerlink" href="#avoiding-getting-banned" title="永久链接至标题">¶</a></h2>
<p>一些网站实施了某些措施，以防止僵尸爬行他们，不同程度的复杂度。绕开这些措施既困难又棘手，有时可能需要特殊的基础设施。请考虑联系 <a class="reference external" href="https://scrapy.org/support/">commercial support</a> 如果有疑问。</p>
<p>以下是处理此类网站时要记住的一些提示：</p>
<ul class="simple">
<li><p>将你的用户代理从浏览器中的一个著名的池中轮换出来（用google搜索以获得一个列表）。</p></li>
<li><p>禁用cookies（请参见 <a class="reference internal" href="downloader-middleware.html#std-setting-COOKIES_ENABLED"><code class="xref std std-setting docutils literal notranslate"><span class="pre">COOKIES_ENABLED</span></code></a> ）因为有些网站可能会使用cookie来发现机器人行为</p></li>
<li><p>使用下载延迟（2或更高）。见 <a class="reference internal" href="settings.html#std-setting-DOWNLOAD_DELAY"><code class="xref std std-setting docutils literal notranslate"><span class="pre">DOWNLOAD_DELAY</span></code></a> 设置。</p></li>
<li><p>如果可能，使用 <a class="reference external" href="http://www.googleguide.com/cached_pages.html">Google cache</a> 获取页面，而不是直接访问站点</p></li>
<li><p>使用一个旋转的IP池。例如，自由 <a class="reference external" href="https://www.torproject.org/">Tor project</a> 或者像这样的付费服务 <a class="reference external" href="https://proxymesh.com/">ProxyMesh</a> . 开源替代方案是 <a class="reference external" href="https://scrapoxy.io/">scrapoxy</a> ，可以将自己的代理附加到的超级代理。</p></li>
<li><p>使用一个在内部绕过BAN的高度分布式下载程序，这样您就可以专注于解析干净的页面。这种下载器的一个例子是 <a class="reference external" href="https://scrapinghub.com/crawlera">Crawlera</a></p></li>
</ul>
<p>如果您仍然无法阻止您的bot被禁止，请考虑联系 <a class="reference external" href="https://scrapy.org/support/">commercial support</a> .</p>
</div>
</div>


           </div>
           
          </div>
          <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="broad-crawls.html" class="btn btn-neutral float-right" title="宽爬行" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="contracts.html" class="btn btn-neutral float-left" title="蜘蛛合约" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
      
    </div>
  

  <hr/>

  <div role="contentinfo">
    <p>
        
        &copy; 版权所有 2008–2020, Scrapy developers
      <span class="lastupdated">
        最后更新于 10月 18, 2020.
      </span>

    </p>
  </div>
    
    
    
    Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a
    
    <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a>
    
    provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
  
 
<script type="text/javascript">
!function(){var analytics=window.analytics=window.analytics||[];if(!analytics.initialize)if(analytics.invoked)window.console&&console.error&&console.error("Segment snippet included twice.");else{analytics.invoked=!0;analytics.methods=["trackSubmit","trackClick","trackLink","trackForm","pageview","identify","reset","group","track","ready","alias","page","once","off","on"];analytics.factory=function(t){return function(){var e=Array.prototype.slice.call(arguments);e.unshift(t);analytics.push(e);return analytics}};for(var t=0;t<analytics.methods.length;t++){var e=analytics.methods[t];analytics[e]=analytics.factory(e)}analytics.load=function(t){var e=document.createElement("script");e.type="text/javascript";e.async=!0;e.src=("https:"===document.location.protocol?"https://":"http://")+"cdn.segment.com/analytics.js/v1/"+t+"/analytics.min.js";var n=document.getElementsByTagName("script")[0];n.parentNode.insertBefore(e,n)};analytics.SNIPPET_VERSION="3.1.0";
analytics.load("8UDQfnf3cyFSTsM4YANnW5sXmgZVILbA");
analytics.page();
}}();

analytics.ready(function () {
    ga('require', 'linker');
    ga('linker:autoLink', ['scrapinghub.com', 'crawlera.com']);
});
</script>


</body>
</html>